package com.splider.toutiao;

import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.apache.log4j.Logger;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.splider.toutiao.mapper.ArticleMapper;
import com.splider.toutiao.mapper.AuthorMaper;

public class TouTiaoSplider {

	// 今日头条首页地址
	public static final String TOUTIAOURL = "http://www.toutiao.com/api/pc/feed/?category=__all__&utm_source=toutiao&widen=1";
	// 头条号主页地址
	public static final String TOUTIAOHAOURL = "http://www.toutiao.com/pgc/ma/?page_type=1&count=10&version=2&platform=pc";
	
	// 头条号主页地址
	public static final String TOUTIAOHAOURL1 = "http://www.toutiao.com/c/user/article/?page_type=1&count=20";

	private static SqlSessionFactory sqlSessionFactory = null;

	private static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm");

	private static SimpleDateFormat sdfYMD = new SimpleDateFormat("yyyy-MM-dd");

	private static int refreshCount = 0;

	private static SqlSession sqlSession = null;

	private static Logger logger = Logger.getLogger("toutiao");

	private static AuthorMaper authorMaper = null;

	private static ArticleMapper articleMapper = null;

	private static String[] categorys = { "__all__", "news_hot", "news_society", "news_entertainment", "news_tech",
			"news_sports", "news_car", "news_finance", "funny", "news_military", "news_world", "news_fashion",
			"news_travel", "news_discovery", "news_baby", "news_regimen", "news_story", "news_essay", "news_game",
			"news_history", "news_food"};

	private static int totalCount = 0;
	
	private static List<ProxyIP> proxyIps = new ArrayList<ProxyIP>();
	
	private static String[] urls = {"item/6371560409240961538/"};
	
	private static List<String> localIps = new ArrayList<String>();
	
	private static String otherCategory = "";
	
	private static Map<String, String> header = new HashMap<String, String>();
	
	private static Map<String, String> header1 = new HashMap<String, String>();
	
	private static int maxCount = 999999;
	
	public static void getLocalIps(){
		String ips = ApplicationConfig.prop.getProperty("localIps");
		if(StringUtils.isNotEmpty(ips)){
			ips = ips.replace("，", ",");
			ips = ips.trim();
			String[] split = ips.split(",");
			for (String ip : split) {
				localIps.add(ip);
			}
		}
	}
	/**
	 * 设置出口ip
	 */
	public static boolean setOutIp(){
		for (String ip : localIps) {
			try {
				Runtime.getRuntime().exec(ApplicationConfig.prop.getProperty("ipshell")+" "+ip);
				logger.info("设置出口ip:"+ip);
				if(checkIpCanUsed(null, null)){
					return true;
				}
			} catch (IOException e) {
				logger.error(e.getMessage());
			}
		}
		return false;
	}

	public static void main(String[] args) {
		String mc = ApplicationConfig.prop.getProperty("max_article_count");
		if(StringUtils.isNotEmpty(mc)){
			mc = mc.trim();
			try {
				maxCount = Integer.parseInt(mc);
			} catch (Exception e) {
				logger.error("max_article_count设置有误");
			}
		}
		//设置请求头
		header.put("Host", "www.toutiao.com");
		header.put("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0");
		header.put("Accept","text/javascript, text/html, application/xml, text/xml, */*");
		header.put("Accept-Language","zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");
		header.put("Accept-Encoding","gzip, deflate");
		header.put("X-Requested-With","XMLHttpRequest");
		header.put("Content-Type","text/*, application/xml");
		header.put("Connection","keep-alive");
		header.put("Cookie", "_ga=GA1.2.1579840638.1479797898; uuid=\"w:1c8b3d677a314defa8b63310a32235e9\"; tt_webid=36837665425; CNZZDATA1259612802=141991597-1479797317-%7C1484033888; csrftoken=58b7438e8257c7699bf7032e0de66ce3; __tasessionId=6s9gr2ktw1484035796735");
		
		
		header1.put("Host", "www.toutiao.com");
		header1.put("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.22 Safari/537.36 SE 2.X MetaSr 1.0");
		header1.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
		header1.put("Accept-Language","zh-CN,zh;q=0.8");
		header1.put("Accept-Encoding","gzip, deflate, sdch");
		header1.put("Content-Type","application/octet-stream");
		header1.put("Connection","keep-alive");
		header1.put("Upgrade-Insecure-Requests", "1");
		header1.put("Cookie", "CNZZDATA1258609184=1974985307-1467773945-null%7C1467773945; csrftoken=e4c83e1812298f85f8e4078dc8b57218; CNZZDATA1259612802=1377273964-1479092337-%7C1480321095; BDTUJIAID=7eabb1c9ec08aff42d58877b65078234; uuid=\"w:bb7ef44dc245484abdc44f6d87ea2ce5\"; install_id=0; tt_webid=52135437746; _ba=BA0.2-20170216-51d9e-LDtGd5em3NsepU2fM7sV; utm_source=toutiao; login_flag=603070cdb797d6bc8f92bd826a5d84a1; sessionid=e7b197b90996ef5572160140f4008667; sid_tt=e7b197b90996ef5572160140f4008667; sid_guard=\"e7b197b90996ef5572160140f4008667|1487235417|2592000|Sat\054 18-Mar-2017 08:56:57 GMT\"; tadseekerHwuid=01934733383571565624160144754383; csrftoken=64cf6d84b869e5dfcb0be5b2474d91b3; __utmt=1; __utma=24953151.1423919142.1473230525.1483683326.1487236733.13; __utmb=24953151.3.10.1487236733; __utmc=24953151; __utmz=24953151.1483683326.12.5.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; cp=58A537E6540F4E1; __tasessionId=w09txf4jg1487235416093; CNZZDATA1259612802=1377273964-1479092337-%7C1487233765; _ga=GA1.2.2132494613.1484701428");
		//获取本地的多个ip
		getLocalIps();
		//是否使用代理ip
		if(!"0".equals(ApplicationConfig.prop.getProperty("useProxy"))){
			//抓取代理ip
			logger.info("抓取代理ip");
			proxyIps = CommonUtils.getProxyIPList();
			logger.info("抓取代理ip结束，总数:"+proxyIps.size());
			//设置代理ip
			if(!setProxyIp()){
				logger.info("没有可用ip，程序退出");
				return;
			}
		}else{
			//设置本机的出口ip
			if(!setOutIp()){
				logger.info("没有可用ip，程序退出");
				return;
			}
		}
		long begin = new Date().getTime();
		sqlSessionFactory = MyBatisUtils.getSqlSessionFactory();
		sqlSession = sqlSessionFactory.openSession();
		authorMaper = sqlSession.getMapper(AuthorMaper.class);
		articleMapper = sqlSession.getMapper(ArticleMapper.class);
		int data[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 };
		int resultdata[] = new int[categorys.length];
		List<Integer> dataArray = new ArrayList<Integer>();
		for (int i = 0; i < data.length; i++) {
			dataArray.add(data[i]);
		}
		int i = 0;
		while (dataArray.size() > 0) {
			int index = (int) (Math.random() * dataArray.size());
			resultdata[i] = dataArray.get(index);
			i++;
			dataArray.remove(index);
		}
		for (int j = 0; j < resultdata.length; j++) {
			logger.info("----------访问栏目:" + categorys[resultdata[j]] + "----------");
			refreshCount = 0;
			otherCategory = categorys[resultdata[j]];
			if(otherCategory.equals("__all__")){
				header.put("Referer","http://www.toutiao.com/");
			}else{
				header.put("Referer","http://www.toutiao.com/"+categorys[resultdata[j]]+"/");
			}
			long time = 0;
			for (; time >= 0;) {
				logger.info("max_behot_time:"+time);
				time = execute(categorys[resultdata[j]], time);
				try {
					Thread.sleep(5000);
				} catch (InterruptedException e) {
				}
			}
		}
		sqlSession.close();
		long end = new Date().getTime();
		logger.info("抓取结束,本次抓取共更新:" + totalCount + "条数据,用时:"+((end-begin)/1000/60)+"分");
	}

	public static long execute(String category, long hottime) {
		if(totalCount>=maxCount){
			return -1;
		}
		refreshCount++;
		logger.info("----------第" + refreshCount + "次刷新----------");
		String url = "http://www.toutiao.com/api/pc/feed/?category=" + category + "&utm_source=toutiao&widen=1"
				+ "&max_behot_time=0&max_behot_time_tmp=" + hottime;
		JSONObject param = null;
		param = CommonUtils.getUrlParam();
		url += "&as=" + param.get("as") + "&cp=" + param.get("cp");
		JSONObject json = null;
		try {
			json = getReturnJson(url);
		} catch (Exception e) {
			e.printStackTrace();
		}
		if (json != null) {
			long time = json.getJSONObject("next").getLongValue("max_behot_time");
			JSONArray data = json.getJSONArray("data");
			if(data==null||data.size()==0){
				return -1;
			}
			for (int i = 0; i < data.size(); i++) {
				try {
					JSONObject obj = (JSONObject) data.get(i);
					String author = obj.getString("source");// 头条号名称
					String authorUrl = obj.getString("media_url");// 头条号主页url
					String authorId = null;
					if(obj.getBooleanValue("has_video")){
						continue;
					}
					if (StringUtils.isNotEmpty(author) && StringUtils.isNotEmpty(authorUrl)) {
						authorId = authorUrl.replace("http://toutiao.com/m", "");
						authorId = authorId.replace("/", "");// 头条号id
						authorId = authorId.replace("m", "");
						authorId = authorId.replace("cuser", "");
						Map<String, Object> authorMap = new HashMap<String, Object>();
						authorMap.put("id", authorId);
						Map<String, Object> authr = authorMaper.selectAuthor(authorMap);
						List<String> topArticle = new ArrayList<String>();
						if (authr != null && authr.size() > 0) {
							Map<String, Object> map = new HashMap<String, Object>();
							map.put("authorid", authorId);
							logger.info("再次访问头条号：" + author + ",url:" + authorUrl);
							long ptime = 0;
							List<String> authorList = new ArrayList<String>();
							List<String> existArts = new ArrayList<String>();
							authorList.add(authorId);
							for (; ptime >= 0;) {
								ptime = connectSubPage(sqlSession, authorMap.get("id").toString(), authorUrl, ptime,authorList,existArts,topArticle);
							}
						} else {
							// 访问头条号主页url
							logger.info("访问头条号：" + author + ",url:" + authorUrl);
							long ptime = 0;
							List<String> authorList = new ArrayList<String>();
							for (; ptime >= 0;) {
								ptime = connectSubPage(sqlSession, authorMap.get("id").toString(), authorUrl, ptime,authorList,null,topArticle);
							}
						}
						sqlSession.commit();
					}
				} catch (Exception e) {
					logger.error("error:" + e.getMessage());
				}
			}
			if(time<=0){
				return -1;
			}else{
				return time;
			}
		}
		return -1;
	}

	public static JSONObject getReturnJson(String url) {
		Connection connect = Jsoup.connect(url);
		Connection con = connect.headers(header);
		con.timeout(Integer.parseInt(ApplicationConfig.prop.getProperty("time_out")));
		try {
			Document doc = con.ignoreContentType(true).get();
			return JSONObject.parseObject(doc.body().text());
		} catch (IOException e2) {
//			logger.error("访问失败:" + url + ",隔2s在访问一次");
//			try {
//				Thread.sleep(2000);
//			} catch (InterruptedException e) {
//				
//			}
//			Connection connect1 = Jsoup.connect(url);
//			Connection con1 = connect1.data(header);
//			try {
//				Document doc = con1.ignoreContentType(true).get();
//				return JSONObject.parseObject(doc.body().text());
//			} catch (IOException e) {
				logger.error("访问失败:" + url);
//			}
		}
		return null;
	}

	private static long connectSubPage(SqlSession sqlSession, String authorId, String authorUrl, long hotTime,List<String> authorList,List<String> existArticles,List<String> topArticles) {
		String media_id = authorUrl.replace("http://toutiao.com/m", "");
		media_id = media_id.replace("/", "");
		media_id = media_id.replace("m", "");
		header1.put("Referer", "http://toutiao.com/m"+media_id);
		JSONObject json = getSubPage(media_id, hotTime);
		if (json != null) {
			long time = json.getJSONObject("next").getLongValue("max_behot_time");
			JSONArray data = json.getJSONArray("data");
			int nullCount = 0;// 访问子页面出错次数，超过3次则不在抓取
			int addCount = 0;
			for (int i = 0; i < data.size(); i++) {
				if (nullCount > 3) {
					sqlSession.commit();
					return -1;
				}
				try {
					JSONObject obj = (JSONObject) data.get(i);
					String articleUrl = obj.getString("source_url");
					String title = obj.getString("title");// 标题
					String articleId = articleUrl.substring(articleUrl.indexOf("item/") + 5);
					articleId = articleId.replace("/", "");
					if (articleId.matches("[0-9]+")) {
						if(topArticles.size()<2){
							//置顶的文章
							topArticles.add(articleId);
						}
						Date publishTime = null;// 发布时间
						Date publishDay = null;
						Date nowTime = null;
						try {
							if(StringUtils.isNotEmpty(obj.getString("datetime"))){
								publishTime = sdf.parse(obj.getString("datetime"));
							}else if(StringUtils.isNotEmpty(obj.getString("behot_time"))){
								publishTime = new Date(obj.getLongValue("behot_time")*1000);
							}
							
						} catch (ParseException e) {
							publishTime = obj.getDate("datatime");
						}
						try {
							Calendar calendar = Calendar.getInstance();
//							calendar.add(Calendar.DAY_OF_MONTH, -1);
							String today = calendar.get(Calendar.YEAR) + "-" + (calendar.get(Calendar.MONTH) + 1) + "-"
									+ calendar.get(Calendar.DAY_OF_MONTH);
							publishDay = sdfYMD.parse(sdf.format(publishTime));
							logger.info("today is " + today);
							logger.info("publishDay is " + publishDay);
							nowTime = sdfYMD.parse(today);
						} catch (ParseException e) {
							publishDay = obj.getDate("datatime");
						}
						long timestamp = publishDay.getTime() / 1000;
						// 是不是今天发布的，置顶的文章跳过
						if(topArticles.size()==1){
							if (timestamp < nowTime.getTime() / 1000) {
								continue;
							}
						}else{
							if (timestamp < nowTime.getTime() / 1000) {
								logger.info("更新的数据已经读取完成,本次更新:"+addCount);
								sqlSession.commit();
								return -1;
							}
						}
						// 排除视频和图册
						if (obj.getBooleanValue("has_video") || obj.getBooleanValue("has_gallery")) {
							continue;
						}
						if(obj.containsKey("display_url ")&&obj.getString("display_url ")!=null&&!obj.getString("display_url ").contains("toutiao.com")){
							sqlSession.commit();
							return -1;
						}
						//文章是否已经抓取过
						Map<String, Object> aMap = new HashMap<String, Object>();
						aMap.put("articleid", articleId);
						List<Map<String, Object>> arts = articleMapper.selectArticle(aMap);
						if(arts!=null&&arts.size()>0){
							existArticles.add(articleId);
							if(existArticles.size()==2){
								sqlSession.commit();
								logger.info("更新的数据已经读取完成,本次更新:"+addCount);
								return -1;
							}else{
								continue;
							}
						}
						String thumbnail = "";
						// 缩略图
						JSONArray imgList = obj.getJSONArray("image_list");
						if (imgList != null && imgList.size() > 0) {
							String prefix = ApplicationConfig.prop.getProperty("thumb_imgUrl_prefix");
							if (StringUtils.isNotEmpty(prefix) && prefix.lastIndexOf("/") != prefix.length() - 1) {
								prefix = prefix + "/";
							}
							String pic_url = imgList.getJSONObject(0).getString("pc_url");
							if(StringUtils.isEmpty(pic_url)){
								pic_url = imgList.getJSONObject(0).getString("url");
							}
							if (StringUtils.isNotEmpty(pic_url)
									&& !"0".equals(ApplicationConfig.prop.getProperty("replace_imgUrl_prefix"))) {
								pic_url = pic_url.substring(pic_url.lastIndexOf("/") + 1);
								pic_url = prefix + pic_url;
							}
							thumbnail = pic_url;
						}
						String tag = null;
						String catagory = null;
						String catagoryFlag = null;
						String content = null;
						String description = null;
						String authorTitle = null;
						String contentText = "";
						if (StringUtils.isNotEmpty(articleUrl)) {
							// 访问文章页面，获取其他信息
							Map<String, Object> articleInfo = getArticleInfo(articleUrl);
							if (articleInfo != null) {
								tag = articleInfo.get("tag") == null ? "" : articleInfo.get("tag").toString();
								catagoryFlag = articleInfo.get("section") == null ? ""
										: articleInfo.get("section").toString();
								content = articleInfo.get("content") == null ? ""
										: articleInfo.get("content").toString();
								contentText = articleInfo.get("contentText") == null ? ""
										: articleInfo.get("contentText").toString();
								description = articleInfo.get("description") == null ? ""
										: articleInfo.get("description").toString();
								authorTitle = articleInfo.get("author") == null ? ""
										: articleInfo.get("author").toString();
								if("其他".equalsIgnoreCase(catagoryFlag)||StringUtils.isEmpty(catagoryFlag)||catagoryFlag.contains("keyword=")){
									catagoryFlag = otherCategory;
								}
								if ("news_hot".equalsIgnoreCase(catagoryFlag)) {// 热点
									catagory = "热点";
								} else if ("news_society".equalsIgnoreCase(catagoryFlag)) {// 社会
									catagory = "社会";
								} else if ("news_entertainment".equalsIgnoreCase(catagoryFlag)) {// 娱乐
									catagory = "娱乐";
								} else if ("news_tech".equalsIgnoreCase(catagoryFlag)) {// 科技
									catagory = "科技";
								} else if ("news_sports".equalsIgnoreCase(catagoryFlag)) {// 体育
									catagory = "体育";
								} else if ("news_car".equalsIgnoreCase(catagoryFlag)) {// 汽车
									catagory = "汽车";
								} else if ("news_finance".equalsIgnoreCase(catagoryFlag)) {// 财经
									catagory = "财经";
								} else if ("news_military".equalsIgnoreCase(catagoryFlag)) {// 军事
									catagory = "军事";
								} else if ("news_world".equalsIgnoreCase(catagoryFlag)) {// 国际
									catagory = "国际";
								} else if ("news_fashion".equalsIgnoreCase(catagoryFlag)) {// 时尚
									catagory = "时尚";
								} else if ("news_travel".equalsIgnoreCase(catagoryFlag)) {// 旅游
									catagory = "旅游";
								} else if ("news_discovery".equalsIgnoreCase(catagoryFlag)) {// 探索
									catagory = "探索";
								} else if ("news_baby".equalsIgnoreCase(catagoryFlag)) {// 育儿
									catagory = "育儿";
								} else if ("news_regimen".equalsIgnoreCase(catagoryFlag)) {// 养生
									catagory = "养生";
								} else if ("news_story".equalsIgnoreCase(catagoryFlag)) {// 故事
									catagory = "故事";
								} else if ("news_essay".equalsIgnoreCase(catagoryFlag)) {// 美文
									catagory = "美文";
								} else if ("news_game".equalsIgnoreCase(catagoryFlag)) {// 游戏
									catagory = "游戏";
								} else if ("news_history".equalsIgnoreCase(catagoryFlag)) {// 历史
									catagory = "历史";
								} else if ("news_food".equalsIgnoreCase(catagoryFlag)) {// 美食
									catagory = "美食";
								} else if("funny".equalsIgnoreCase(catagoryFlag)){
									catagory = "搞笑";
								}else if ("news_health".equalsIgnoreCase(catagoryFlag)) {// 其他
									catagory = "健康";
								}else{
									catagoryFlag = "other";
									catagory = "其他";
								}
								catagoryFlag = catagoryFlag.replace("news_", "");
								// 将头条号信息插入表中
								if (authorList==null||authorList.size()==0) {
									Map<String, Object> map = new HashMap<String, Object>();
									map.put("id", authorId);
									map.put("title", CommonUtils.simple2tw(authorTitle));
									map.put("description", CommonUtils.simple2tw(description));
									authorMaper.insertAuthor(map);
									logger.info("插入头条号作者成功");
									authorList.add(authorId);
								}
							} else {
								nullCount++;
							}
						}
						if (StringUtils.isNotEmpty(content) && StringUtils.isNotEmpty(contentText)
								&& contentText.length() >= 250) {
							//查询文章是否已存在，如果已存在，则不插入，如果连续两篇已存在，则返回
							Map<String, Object> articleMap = new HashMap<String, Object>();
							articleMap.put("articleid", articleId);
							articleMap.put("title", CommonUtils.simple2tw(title));
							articleMap.put("catagory", CommonUtils.simple2tw(catagory));
							articleMap.put("catagoryflag", catagoryFlag);
							content = content.replace("本文为头条号作者发布，不代表今日头条立场。", "");
							content = content.replace("本文为头条号作者原创，未经授权，不得转载。", "");
							String simple2tw = CommonUtils.simple2tw(CommonUtils.encodeHtml(content));
							content = CommonUtils.decodeHtml(simple2tw);
							// 下载html中的图片
							content = downloadPic(content);
							articleMap.put("content", content);
							articleMap.put("tags", CommonUtils.simple2tw(tag));
							articleMap.put("author", CommonUtils.simple2tw(authorTitle));
							articleMap.put("authorId", authorId);
							articleMap.put("url", articleUrl);
							articleMap.put("publishtime", publishTime);
							articleMap.put("thumbnail", thumbnail);
							articleMap.put("date", timestamp);
							articleMapper.insertArticle(articleMap);
							addCount++;
							if(totalCount>=maxCount){
								sqlSession.commit();
								return -1;
							}
							totalCount++;
						}

					}
				} catch (Exception e) {
					logger.error("error:" + e.getMessage());
				}
			}
			sqlSession.commit();
			if(addCount>0){
				logger.info("读取更新:" + addCount + "条");
			}
			if(json.getIntValue("has_more") == 1){
				return time;
			}
		} else {
			logger.info("该头条号读取完成,max_behot_time:" + hotTime);
		}
		sqlSession.commit();
		return -1;
	}

	public static String downloadPic(String html) {
		String prefix = ApplicationConfig.prop.getProperty("imgUrl_prefix");
		if (StringUtils.isNotEmpty(prefix) && prefix.lastIndexOf("/") != prefix.length() - 1) {
			prefix = prefix + "/";
		}
		Document document = Jsoup.parse(html);
		Elements imgs = document.getElementsByTag("img");
		if (imgs != null && imgs.size() > 0) {
			for (Element img : imgs) {
				String imgUrl = img.attr("src");
				String imgName = download(imgUrl);
				if (StringUtils.isNotEmpty(imgName)) {
					img.attr("src", prefix + imgName);
				}
			}
		}
		if ("0".equals(ApplicationConfig.prop.getProperty("replace_imgUrl_prefix"))) {
			// 不修改url前缀
			return html;
		} else {
			return document.getElementsByTag("body").html();
		}
	}

	public static String download(String imgUrl) {
		try {
			String imgName = imgUrl.substring(imgUrl.lastIndexOf("/") + 1);
			if ("1".equals(ApplicationConfig.prop.getProperty("download_img"))) {
				// 下载图片
				URL url = new URL(imgUrl);
				HttpURLConnection conn = (HttpURLConnection) url.openConnection();
				conn.setConnectTimeout(Integer.parseInt(ApplicationConfig.prop.getProperty("time_out")));
				conn.setReadTimeout(Integer.parseInt(ApplicationConfig.prop.getProperty("time_out")));
				DataInputStream dataInputStream = new DataInputStream(conn.getInputStream());
				imgName = imgName + ".jpg";
				File dir = new File(ApplicationConfig.prop.getProperty("img_dir"));
				if (!dir.exists()) {
					dir.mkdirs();
				}
				File file = new File(ApplicationConfig.prop.getProperty("img_dir") + File.separator + imgName);
				if (!file.exists()) {
					file.createNewFile();
				}
				FileOutputStream fileOutputStream = new FileOutputStream(file);
				byte[] buffer = new byte[1024];
				int length;
				while ((length = dataInputStream.read(buffer)) > 0) {
					fileOutputStream.write(buffer, 0, length);
				}
				dataInputStream.close();
				fileOutputStream.close();
			}
			return imgName;
		} catch (MalformedURLException e) {
			logger.error("error:" + e.getMessage());
		} catch (IOException e) {
			logger.error("error:" + e.getMessage());
		}
		return null;
	}

	public static JSONObject getSubPage(String media_id, long hotTime) {
		String url = TOUTIAOHAOURL + "&media_id=" + media_id + "&max_behot_time=" + hotTime;
		if(media_id.contains("user")){
			url = TOUTIAOHAOURL1 + "&user_id="+media_id.replace("cuser", "")+"&max_behot_time="+hotTime;
		}
		JSONObject param = null;
		param = CommonUtils.getUrlParam();
		url += "&as=" + param.get("as") + "&cp=" + param.get("cp");
		logger.info("sub url:"+url);
		try {
			URL httpUrl = new URL(url);
			httpUrl.openConnection().setConnectTimeout(Integer.parseInt(ApplicationConfig.prop.getProperty("time_out")));
			httpUrl.openConnection().setReadTimeout(Integer.parseInt(ApplicationConfig.prop.getProperty("time_out")));
			BufferedReader in = new BufferedReader(new InputStreamReader(httpUrl.openStream(), "UTF-8"));
			String line = null;
			StringBuffer content = new StringBuffer("");
			while ((line = in.readLine()) != null) {
				content.append(line);
			}
			in.close();
			line = null;
			in = null;
			httpUrl = null;
			return JSONObject.parseObject(content.toString());
		} catch (Exception e) {
//			logger.error("访问子页面失败:" + url + ",隔2s再访问一次");
//			try {
//				Thread.sleep(2000);
//				URL httpUrl = new URL(url);
//				BufferedReader in = new BufferedReader(new InputStreamReader(httpUrl.openStream(), "UTF-8"));
//				String line = null;
//				StringBuffer content = new StringBuffer("");
//				while ((line = in.readLine()) != null) {
//					content.append(line);
//				}
//				in.close();
//				line = null;
//				in = null;
//				httpUrl = null;
//				return JSONObject.parseObject(content.toString());
//			} catch (Exception e1) {
				logger.error("访问子页面失败:" + url);
//			}
		}
		return null;
	}

	public static Map<String, Object> getArticleInfo(String url) {
		if(!url.contains("toutiao.com")){
			url = "http://www.toutiao.com"+url;
		}
		try {
			Map<String, Object> map = new HashMap<String, Object>();
			Connection connect = Jsoup.connect(url).headers(header1);
			Document document;
			document = connect.timeout(Integer.parseInt(ApplicationConfig.prop.getProperty("time_out"))).get();
			Element header = document.getElementById("header");
			Elements divs = header.getElementsByClass("chinese-tag");
			String section = "";
			if (divs != null && divs.size() > 0) {
				if (divs.get(0).getElementsByTag("a").size() > 1) {
					String href = divs.get(0).getElementsByTag("a").get(1).attr("href");
					if (StringUtils.isNotEmpty(href)) {
						section = href.replace("/", "");
					} else {
						section = "其他";
					}
				}
			}
			Elements tagElements = document.getElementsByClass("label-list");
			Elements tags = null;
			if (tagElements != null && tagElements.size() > 0) {
				tags = tagElements.get(0).getElementsByTag("a");
			}
			String tag = "";
			StringBuffer sb = new StringBuffer();
			if (tags != null) {
				for (Element t : tags) {
					sb.append(t.text()).append(",");
				}
				if (StringUtils.isNotEmpty(sb.toString())) {
					tag = sb.deleteCharAt(sb.lastIndexOf(",")).toString();
				}
			}
			Elements contentEle = document.getElementsByClass("article-content");
			String content = "";
			String contentText = "";
			if (contentEle != null && contentEle.size() > 0) {
				content = contentEle.get(0).html();
				contentText = contentEle.get(0).text();
			}
			String description = "";
			Elements descEle = document.getElementsByClass("pgc-description");
			if (descEle != null && descEle.size() > 0) {
				description = descEle.get(0).text();
			}
			Elements userEle = document.getElementsByClass("pgc-username");
			String author = "";
			if (userEle != null && userEle.size() > 0) {
				Elements span = userEle.get(0).getElementsByTag("span");
				if (span != null && span.size() > 0) {
					author = span.get(0).text();
				}
			}
			map.put("author", author);
			map.put("description", description);
			map.put("content", content);
			map.put("contentText", contentText);
			map.put("section", section);
			map.put("tag", tag);
			return map;
		} catch (IOException e) {
			logger.error("访问文章页失败:" + url + "  原因" + e.getMessage());
		}
		return null;
	}
	
	public static boolean setProxyIp(){
		//取1个页面访问，如果都访问不了的话，则换ip，如果都能访问，则设置为代理ip，如果都没有的话则使用本机ip
		if(proxyIps!=null&&proxyIps.size()>0){
			Iterator<ProxyIP> it = proxyIps.iterator();
			while(it.hasNext()){
				ProxyIP ip = it.next();
				logger.info("测试ip是否可用："+ip.getIp());
				if(checkIpCanUsed(ip.getIp(), ip.getPort()+"")){
					logger.info("代理IP："+ip.getIp()+"可用");
					return true;
				}else{
					it.remove();
				}
			}
		}
		System.getProperties().setProperty("http.proxyHost", null);
		System.getProperties().setProperty("http.proxyPort", null);
		//先判断本机ip能否访问
		if(setOutIp()){
			return true;
		}
		return false;
	}
	
	public static boolean checkIpCanUsed(String ip,String port){
		if(StringUtils.isNotEmpty(ip)){
			System.getProperties().setProperty("http.proxyHost", ip);
		}
		if(StringUtils.isNotEmpty(port)){
			System.getProperties().setProperty("http.proxyPort", port);
		}
		boolean b = false;
		boolean[] b1 = new boolean[urls.length];
		logger.info("测试访问栏目：__all__");
		String url = "http://www.toutiao.com/api/pc/feed/?category=__all__&utm_source=toutiao&widen=1"
				+ "&max_behot_time=0&max_behot_time_tmp=0";
		JSONObject param = null;
		param = CommonUtils.getUrlParam();
		url += "&as=" + param.get("as") + "&cp=" + param.get("cp");
		JSONObject json = null;
		try {
			json = getReturnJson(url);
		} catch (Exception e) {
			e.printStackTrace();
		}
		if(json!=null&&json.getJSONArray("data")!=null&&json.getJSONArray("data").size()>0){
			b = true;
		}else{
			b = false;
			return false;
		}
		
		boolean s = false;
		logger.info("测试访问头条号：52155491628");
		JSONObject subPage = getSubPage("52155491628", 0);
		try {
			if(subPage!=null&&subPage.getJSONArray("data")!=null&&subPage.getJSONArray("data").size()>0){
				s = true;
			}else{
				s = false;
				return false;
			}
		} catch (Exception e) {
			s = false;
		}
		for (int j=0; j < urls.length;j++) {
			logger.info("测试访问页面："+urls[j]);
			Map<String, Object> info = getArticleInfo("http://www.toutiao.com/"+urls[j]+"/");
			if(info!=null){
				b1[j] = true;
			}else{
				b1[j] = false;
			}
			j++;
		}
		int count1 = 0;
		for (boolean c : b1) {
			if(c){
				count1++;
			}
		}
		if(b&&count1==urls.length&&s){
			return true;
		}else{
			return false;
		}
	}
}
